package org.candan.spider.task.spider;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.Map;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;
import org.candan.spider.bean.DataSift;
import org.candan.spider.bean.Plan;
import org.candan.spider.bean.TagSift;
import org.candan.spider.task.Container;
import org.candan.spider.util.DateUtil;
import org.candan.spider.util.SLog;



public class HttpDownload  extends SpiderPlan{
	public HttpDownload(Plan plan) {
		super(plan);
		// TODO Auto-generated constructor stub
	}
	public HttpDownload(Plan plan,DataSift dst, TagSift tsf){
		super(plan,dst,tsf);
	}
	private Logger logger = new SLog("HttpDownload").getLogger();
	private int conNum = 1;

	@Override
	protected void execueSpiderPlan(String url, String output){
		connect(url, output,null);
	}
	
	/**
	 * @category http下载
	 * @param url http网址
	 * @param outPutDir 下载目录
	 * @param filter 
	 * @return
	 */
	public boolean connect(String url, String outPutDir,String filter){
		Map<String, String> links = tagExtract();
		if(!(outPutDir.endsWith("/") || outPutDir.endsWith("\\"))){
			outPutDir+= "/";
		}
		for(Map.Entry<String, String> set:links.entrySet()){
			if(!dataFilterDownNumCheck())break;
			if(!dataFilterBasicCheck(set.getKey())) continue;
			downloadFile(set.getValue(),outPutDir +set.getKey());
			
		}
		return true;
	}
	public boolean connect2(String url, String outPutDir,String filter){
		if(!(outPutDir.endsWith("/") || outPutDir.endsWith("\\"))){
			outPutDir+= "/";
		}
		String file = url.substring(url.lastIndexOf("/")+1);
		if(!dataFilterBasicCheck(file)) return false;
		downloadFile(url,outPutDir +file);
		return true;
	}
	/**
	 * @category 下载文件
	 * @param url
	 * @param outPutFile
	 * @return
	 */
	@SuppressWarnings("deprecation")
	public boolean downloadFile(String url, String outPutFile){
		HttpClient httpClient = new DefaultHttpClient();  
		HttpGet httpGet = new HttpGet(url);   
		File storeFile = new File(outPutFile);
		 if(!lockFile(storeFile)){
	        	return false;
	        }
		try {
			HttpResponse response = httpClient.execute(httpGet);
			if(HttpStatus.SC_OK==response.getStatusLine().getStatusCode()){   
	            //请求成功   取得请求内容   
	            HttpEntity entity = response.getEntity();  
	            //显示内容   
	            if (entity != null && entity.isStreaming()) {   
	            	long localSize = 0L;
	            	long lRemoteSize = entity.getContentLength();
	            	long step = lRemoteSize /100;  
	            	byte b[] = new byte[1024];   
	            	int j = 0;  
	            	long process=0L; 
	            	long nowProcess=0L;
	            	String fileSize = DateUtil.getNumFmat(lRemoteSize/1000.0)+"KB";
	            	InputStream input = entity.getContent();  
	            	FileOutputStream output = null;   
//	            	if(storeFile.exists() && storeFile.length() >= lRemoteSize){
//	            		return false;
//	            	}
	            	if(isExistWholeFile(storeFile,lRemoteSize)){
//	                	return DownloadStatus.Local_Bigger_Remote;
	            		return false;
	                }
	            	logger.info("==开始下载："+storeFile.getName()+" ,  "+fileSize+"     保存到："+storeFile.getAbsolutePath()+"   网址:"+url);
	            	if(storeFile.exists()){
	            		localSize = storeFile.length();
	            		output = new FileOutputStream(storeFile,true);
	            		long tmpSize = 0L;
	            		boolean toDirection = false;
	            		//得到网络资源并写入文件   
	            		while( (j = input.read(b))!=-1){
	            			if(tmpSize + j <= localSize){
	            				tmpSize += j;
	            				continue;
	            			}else if(!toDirection){
	            				toDirection = true;
//	            				System.out.println(localSize-tmpSize +"  , "+j);
	            				output.write(b,(int) (localSize-tmpSize), (int)(j + tmpSize - localSize));  
	            				tmpSize += j;
	            			}else{
		            			output.write(b,0,j);   
		            			tmpSize += j;
		            			nowProcess = tmpSize /step; 
		            			if(nowProcess > process){   
		                            process = nowProcess;   
		                            if(process % 10 == 0){
		                            	logger.info("    "+storeFile.getName()+":下载进度："+process+"%");   
		                            }
		                            //TODO 更新文件下载进度,值存放在process变量中   
		                        } 
	            			}
	            		}   
	            		output.flush();   
	            		output.close();    
	            	}else{
	            		output = new FileOutputStream(storeFile);
	            		while( (j = input.read(b))!=-1){   
	            			output.write(b,0,j);   
	            			localSize += j;
	            			nowProcess = localSize /step; 
	            			if(nowProcess > process){   
	                            process = nowProcess;   
	                            if(process % 10 == 0){
	                            	logger.info("    "+storeFile.getName()+":下载进度："+process+"%");   
	                            }
	                            //TODO 更新文件下载进度,值存放在process变量中   
	                        } 
	            		}   
	            		output.flush();   
	            		output.close();  
	            	}
	            	
	            	addResult(storeFile.getName(), fileSize, storeFile.getAbsolutePath());
	            	logger.info("==完成下载:"+storeFile.getName());
	            }   
	            if (entity != null) {   
	                entity.consumeContent();   
	            }   
	        }else{
	        	if(++conNum <=20){
	        		printMsg(new Exception("。。。请求失败！"));
					downloadFile(url, outPutFile);
				  }else{
			          conNum = 1;
			          logger.info("下载进度失败"); 
			          return false;
				  }
	        }
		} catch (IOException e) {
			if(++conNum <=20){
				printMsg(e);
				downloadFile(url, outPutFile);
			  }else{
		          e.printStackTrace();
		          conNum = 1;
		          logger.info("下载进度失败"); 
		          return false;
			  }
		}finally{
			releaseLock(storeFile);
		}
		return true;
	}
	
//	/**
//	 * @category 解析需下载文件
//	 * @param url
//	 * @param outPutDir
//	 * @param filters
//	 * @return
//	 */
//	public Map<String,String> parseFiles(String url, String outPutDir,String... filters){
//		logger.info("解析url,准备http下载："+url);
//		Map<String,String> links = new HashMap<String,String>();
//		try {
//			Parser parser = new Parser(url);
//			parser.setEncoding("gb2312");
//			NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);
//			boolean makeSure = true;
//			// 得到所有经过过滤的标签
//			NodeList list = parser.extractAllNodesThatMatch(linkFilter);
//			for (int i = 0; i < list.size(); i++) {
//				Node tag = list.elementAt(i);
//				if (tag instanceof LinkTag)// <a> 标签
//				{
//					LinkTag link = (LinkTag) tag;
//					//如果link没有href引用，则跳过
//					if(!link.getText().contains("href"))continue;
//					//过滤条件
//					for(String s: filters){
//						if(!link.getAttribute("href").contains(s)){
//							makeSure = false;
//							break;
//						}
//					}
//					if(!makeSure){
//						makeSure = true;
//						continue;
//					}
//					links.put(link.getAttribute("href"), link.getLink());
//				} 
//			}
//		} catch (ParserException e) {
//			 if(++conNum <=20){
//				printMsg(e);
//				parseFiles(url, outPutDir);
//			  }else{
//		          e.printStackTrace();
//		          conNum = 1;
//		          return null;
//			  }
//		}
//		 conNum = 1;
//		return links;
//	}
	/**
	 * @category 打印出错信息
	 * @param e
	 */
	private void printMsg(Exception e){
		 logger.error("连接http出错："+e.getMessage()+"; 进入第" + conNum+" 次连接！共尝试 20 次连接");
		  try {
			Thread.sleep(2000);
		} catch (InterruptedException e1) {
			e1.printStackTrace();
		}
	}
	  public static void main(String[] args){
		  DataSift dsf = new DataSift();
		  dsf.setRegex("yyyyMMdd");
		  dsf.setRange(5);
		  dsf.setMaxDownload(5);
		  dsf.setFromDate(new Date(2012,01,01));
		  dsf.setToDate(new Date(2025,12,12));
		  new HttpDownload(null,dsf,null).connect2("http://www.temis.nl/airpollution/no2col/data/gome2_v2/2012/11/no2track20121118.hdf", "d:/temp",null);
	  }
}
